{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR10 Image Classification with Vision in Transformers (ViT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, random_split, DataLoader\n",
    "\n",
    "from torchvision import datasets, transforms, models\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import Dataset, random_split, DataLoader\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import time\n",
    "import math\n",
    "from PIL import Image\n",
    "import glob\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "LR = 3e-4\n",
    "NUM_EPOCHES = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean, std = (0.5,), (0.5,)\n",
    "\n",
    "transform = transforms.Compose([transforms.ToTensor(),\n",
    "                                transforms.Normalize(mean, std)\n",
    "                              ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainset = datasets.CIFAR10('data/CIFAR10/', download=True, train=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "\n",
    "testset = datasets.CIFAR10('data/CIFAR10/', download=True, train=False, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer_package.models import ViT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ViT(\n",
       "  (dropout_layer): Dropout(p=0.2, inplace=False)\n",
       "  (embeddings): Linear(in_features=192, out_features=512, bias=True)\n",
       "  (encoders): ModuleList(\n",
       "    (0): VisionEncoder(\n",
       "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (attention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.2, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (mlp): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=2048, bias=True)\n",
       "        (1): GELU()\n",
       "        (2): Dropout(p=0.2, inplace=False)\n",
       "        (3): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        (4): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (1): VisionEncoder(\n",
       "      (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (attention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.2, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (mlp): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=2048, bias=True)\n",
       "        (1): GELU()\n",
       "        (2): Dropout(p=0.2, inplace=False)\n",
       "        (3): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        (4): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=512, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_size = 32\n",
    "channel_size = 3\n",
    "patch_size = 8\n",
    "embed_size = 512\n",
    "num_heads = 8\n",
    "classes = 10\n",
    "num_layers = 2\n",
    "hidden_size = 256\n",
    "dropout = 0.2\n",
    "\n",
    "model = ViT(image_size, channel_size, patch_size, embed_size, num_heads, classes, num_layers, hidden_size, dropout=dropout).to(device)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Image Dimensions: torch.Size([64, 3, 32, 32])\n",
      "Label Dimensions: torch.Size([64])\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Output Dimensions: torch.Size([64, 10])\n"
     ]
    }
   ],
   "source": [
    "for img, label in trainloader:\n",
    "    img = img.to(device)\n",
    "    label = label.to(device)\n",
    "    \n",
    "    print(\"Input Image Dimensions: {}\".format(img.size()))\n",
    "    print(\"Label Dimensions: {}\".format(label.size()))\n",
    "    print(\"-\"*100)\n",
    "    \n",
    "    out = model(img)\n",
    "    \n",
    "    print(\"Output Dimensions: {}\".format(out.size()))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss()\n",
    "optimizer = torch.optim.Adam(params=model.parameters(), lr=LR, betas=(0.9, 0.999), weight_decay=0.1)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------\n",
      "Epoch: 1 Train mean loss: 1789.94122672\n",
      "       Train Accuracy%:  13.152 == 6576 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 2 Train mean loss: 1779.82735944\n",
      "       Train Accuracy%:  13.496 == 6748 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 3 Train mean loss: 1777.52187300\n",
      "       Train Accuracy%:  13.426 == 6713 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 4 Train mean loss: 1777.67711735\n",
      "       Train Accuracy%:  13.56 == 6780 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 5 Train mean loss: 1779.43262100\n",
      "       Train Accuracy%:  13.494 == 6747 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 6 Train mean loss: 1778.38365579\n",
      "       Train Accuracy%:  13.588 == 6794 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 7 Train mean loss: 1775.79081059\n",
      "       Train Accuracy%:  13.784 == 6892 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 8 Train mean loss: 1774.46977425\n",
      "       Train Accuracy%:  14.446 == 7223 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 9 Train mean loss: 1772.39552212\n",
      "       Train Accuracy%:  14.652 == 7326 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 10 Train mean loss: 1771.64754677\n",
      "       Train Accuracy%:  14.738 == 7369 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 11 Train mean loss: 1771.42214131\n",
      "       Train Accuracy%:  14.68 == 7340 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 12 Train mean loss: 1771.05796337\n",
      "       Train Accuracy%:  14.73 == 7365 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 13 Train mean loss: 1770.99634624\n",
      "       Train Accuracy%:  14.916 == 7458 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 14 Train mean loss: 1770.54076791\n",
      "       Train Accuracy%:  14.9 == 7450 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 15 Train mean loss: 1770.25097370\n",
      "       Train Accuracy%:  15.158 == 7579 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 16 Train mean loss: 1770.16693020\n",
      "       Train Accuracy%:  15.064 == 7532 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 17 Train mean loss: 1769.45428419\n",
      "       Train Accuracy%:  14.938 == 7469 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 18 Train mean loss: 1769.20178390\n",
      "       Train Accuracy%:  14.86 == 7430 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 19 Train mean loss: 1763.38752985\n",
      "       Train Accuracy%:  15.13 == 7565 / 50000\n",
      "-------------------------------------------------\n",
      "-------------------------------------------------\n",
      "Epoch: 20 Train mean loss: 1747.16193008\n",
      "       Train Accuracy%:  16.722 == 8361 / 50000\n",
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "loss_hist = {}\n",
    "loss_hist[\"train accuracy\"] = []\n",
    "loss_hist[\"train loss\"] = []\n",
    "\n",
    "for epoch in range(1, NUM_EPOCHES+1):\n",
    "    model.train()\n",
    "    \n",
    "    epoch_train_loss = 0\n",
    "        \n",
    "    y_true_train = []\n",
    "    y_pred_train = []\n",
    "        \n",
    "    for batch_idx, (img, labels) in enumerate(trainloader):\n",
    "        img = img.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        preds = model(img)\n",
    "        \n",
    "        loss = criterion(preds, labels)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        y_pred_train.extend(preds.detach().argmax(dim=-1).tolist())\n",
    "        y_true_train.extend(labels.detach().tolist())\n",
    "            \n",
    "        epoch_train_loss += loss.item()\n",
    "    \n",
    "    #scheduler.step(epoch_train_loss)\n",
    "    \n",
    "    loss_hist[\"train loss\"].append(epoch_train_loss)\n",
    "    \n",
    "    total_correct = len([True for x, y in zip(y_pred_train, y_true_train) if x==y])\n",
    "    total = len(y_pred_train)\n",
    "    accuracy = total_correct * 100 / total\n",
    "    \n",
    "    loss_hist[\"train accuracy\"].append(accuracy)\n",
    "    \n",
    "    print(\"-------------------------------------------------\")\n",
    "    print(\"Epoch: {} Train mean loss: {:.8f}\".format(epoch, epoch_train_loss))\n",
    "    print(\"       Train Accuracy%: \", accuracy, \"==\", total_correct, \"/\", total)\n",
    "    print(\"-------------------------------------------------\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAonklEQVR4nO3deXxU9b3/8dcngbATlgTCFvZFDYgYcUGtS1WgfVRbe6vW1r2oj2qvt7eL3v5+bW+321bb2v7sZhUVW6n2qtUqbrXWBRcIyBJI2AMEEpKwJQESsnx+f8yMpnECAXLmzJD38/HIIzNnmfnkMOGdc873fI65OyIiIq2lhV2AiIgkJwWEiIjEpYAQEZG4FBAiIhKXAkJEROLqEnYBHSkrK8tHjRoVdhkiIiljyZIlVe6eHW/ecRUQo0aNoqCgIOwyRERShpltbmueDjGJiEhcCggREYlLASEiInEpIEREJC4FhIiIxKWAEBGRuBQQIiISlwJCRCSF/X31Dn7/+gaCuHWDAkJEJIU9s3w7j767GTPr8NdWQIiIpLCismpOGNI3kNdWQIiIpKi6hiY2VtZyQk6fQF5fASEikqLW7ail2dEehIiI/Kui8moAJikgRESkpaKyanp0TWfkgJ6BvL4CQkQkRRWX1TAxpw9paR0/ggkUECIiKcndKSoPbgQTKCBERFLSjup69uxv4IQhwYxgggADwszmmlmFmRW2mn67ma0xs1Vm9tM21i0xs5VmtszMdIs4EZFWisoiJ6iD3IMI8pajDwP3AfNiE8zsfOBSYIq715vZoEOsf767VwVYn4hIylodDYiJAV0DAQHuQbj7G8CuVpNvBX7s7vXRZSqCen8RkeNZcXkNw/v3oG/3roG9R6LPQUwAzjGz98zsdTM7rY3lHHjZzJaY2ZwE1icikhKKyqqZlBPc4SUI9hBTW+/XHzgDOA14wszG+EfbEM5w9+3RQ1CvmFlxdI/kI6IBMgcgNzc3wNJFRJJDrMXG7LycQN8n0XsQpcBTHrEIaAayWi/k7tuj3yuAp4Hpbb2gu9/v7vnunp+dnR1Q2SIiySPWYiOoK6hjEh0QfwUuADCzCUAG8C8nos2sl5n1iT0GLgYKERER4MMWG0GOYIJgh7nOB94BJppZqZndCMwFxkSHvv4ZuNbd3cyGmtmC6KqDgbfMbDmwCHje3V8Mqk4RkVQTdIuNmMDOQbj7VW3M+kKcZbcDs6OPNwInB1WXiEiqC7rFRoyupBYRSSGJaLERo4AQEUkhiWixEaOAEBFJIYlosRGjgBARSSGxEUxBttiIUUCIiKSQorLgW2zEKCBERFJIcQJabMQoIEREUkRdQxMbq/ZxYgJOUIMCQkQkZayvqKWp2QNvsRGjgBARSRGrEziCCRQQIiIpo7ishh5d08kNuMVGjAJCRCRFFJVVMzGnD+kBt9iIUUCIiKQAd6e4vDohV1DHKCBERFLAjup6du9vSNj5B1BAiIikhNgV1Im6BgIUECIiKSHWg2mSDjGJiEhLRWU1DOuXmBYbMQoIEZEUUFyWmHtAtKSAEBFJcrEWG4kcwQQKCBGRpBdrsXHc7EGY2VwzqzCzwlbTbzezNWa2ysx+2sa6M6PLrDezO4OqUUQkFcRabExKwD0gWgpyD+JhYGbLCWZ2PnApMMXdTwLuab2SmaUDvwZmAScCV5nZiQHWKSKS1GItNkYO7JXQ9w0sINz9DWBXq8m3Aj929/roMhVxVp0OrHf3je5+EPgzkVAREemUisqqmZDAFhsxiT4HMQE4x8zeM7PXzey0OMsMA7a2eF4anRaXmc0xswIzK6isrOzgckVEwhVrsZGoe0C0lOiA6AL0B84Avg48YWatIzFeRHpbL+ju97t7vrvnZ2dnd1ylIiJJIIwWGzGJDohS4CmPWAQ0A1lxlhnR4vlwYHuC6hMRSSphtNiISXRA/BW4AMDMJgAZQFWrZRYD481stJllAFcCzyaySBGRZBFGi42YIIe5zgfeASaaWamZ3QjMBcZEh77+GbjW3d3MhprZAgB3bwRuA14CioAn3H1VUHWKiCSz4hBabMR0CeqF3f2qNmZ9Ic6y24HZLZ4vABYEVJqISMooCqHFRoyupBYRSVJhtdiIUUCIiCSpsFpsxCggRESSVFFILTZiFBAiIkmqKKQWGzEKCBGRJFVcHk6LjRgFhIhIEnJ3isrCabERo4AQEUlCFTWRFhthXEEdo4AQEUlCsXtAhDWCCRQQIiJJKTaCaWJII5hAASEikpRiLTYyeyS+xUaMAkJEJAlFWmyEt/cACggRkaTzYYuN8M4/gAJCRCTpxFpshDmCCRQQIiJJp+iDEUw6xCQiIi0UldXQvWtaaC02YhQQIiJJpri8mok5fUNrsRGjgBARSSKxFhsnhHj9Q4wCQkQkicRabIQ9ggkUECIiSWV1yPeAaCmwgDCzuWZWYWaFLaZ918y2mdmy6NfsNtYtMbOV0WUKgqpRRCTZFJfVADApCfYgugT42g8D9wHzWk3/hbvf0471z3f3qg6vSkQkiRWVVYfeYiMmsD0Id38D2BXU64uIHI+Ky8NvsRETxjmI28xsRfQQVP82lnHgZTNbYmZzDvViZjbHzArMrKCysrLjqxURSZC6hiY2VIbfYiMm0QHxW2AsMBUoA37WxnIz3H0aMAv4spmd29YLuvv97p7v7vnZ2dkdXa+ISMIkS4uNmIQGhLvvcPcmd28G/gBMb2O57dHvFcDTbS0nInI8SZYWGzEJDQgzG9Li6aeBwjjL9DKzPrHHwMXxlhMROd4UlydHi42YwEYxmdl84Dwgy8xKge8A55nZVCLnGEqAm6PLDgUecPfZwGDgaTOL1feYu78YVJ0iIsmiqCw5WmzEBBYQ7n5VnMkPtrHsdmB29PFG4OSg6hIRSUaxFhuXnJQTdikf0JXUIiJJIJlabMQoIEREkkBRErXYiFFAiIgkgaIkarERo4AQEUkCydRiI0YBISKSBJKpxUaMAkJEJGSxFhvJcgV1jAJCRCRksRYbyTSCCRQQIiKh+2AEkw4xiYhIS7EWG6OSpMVGjAJCRCRkRWXVTBzcJ2labMQoIEREQhRrsZFs5x9AASEiEqpYi41kuoI6Jsh7UouIHFJ9YxPffXY1q8uqueiEQczMG8K4Qb3DLiuhPrwHRPLtQSggRCQUew80cPOjBby7cRcnDOnLPS+v5Z6X1zJ+UG9m5eUwa/IQJuX0Idr6/7j1QYuNJLsGAhQQIhKC8r11XPfQIjZU1nLvFVO57JRhlO09wEuF5bxQWM59r63nV/9Yz6iBPZmZN4RZeTlMGZ553IVFcXk1cxduYmx2LzJ7Jk+LjRhz97Br6DD5+fleUFAQdhkicghrd9Rw3dxFVNc18rsvnMrZ47M+skxlTT0vry7nxcJy3t6wk6ZmZ1i/HszMy2FWXg7TcvuTlmQjfo7Uks27uf6hRfTM6MIfb5rOuEHhnIMwsyXunh93ngJCRBJl0aZd3PTIYrp1Tefh60/jpKGZh11nz/6DvLJ6By8WlvPmuioONjUzqE83LjkpEhbTRw+gS3pqjbd5c10lc+YtYXDfbjx64+mMGNAztFqOOSCi94Y+4O7NZjYBmAS84O4NHVvqsVFAiCSvF1aW8e+PL2N4/x48cv30o/pPsaaugX8UV/DCynL+ubaCuoZmBvTK4BOTh3DtWSND+yv8SLxYWMZX5i9jTHYv5t04nUF9uodaT0cExBLgHKA/8C5QAOx396sPsc5c4JNAhbvnRad9F/gSUBld7L/cfUGcdWcCvwTSidyr+seHLRIFhEiyeuTtEr77t1WcMqIfD157Gv17ZRzza+4/2MjraypZUFjOS6vKOdjYzLkTsrl+xig+Nj47KQ9BPVGwlTufXMEpuf2Ze91pSdHauyMCYqm7TzOz24Ee7v5TM3vf3U85xDrnArXAvFYBUevu9xxivXRgLXARUAosBq5y99WHq1MBIZJcmpudn760ht+9voGLThzMr648hR4Z6R3+Pjtr63nsvS3Me3czlTX1jMnuxfUzRnP5tGH0zEiOsTgPvLmRHzxfxDnjs/j9F09NmroOFRDtPXBnZnYmcDXwfHTaIX86d38D2NXuKj80HVjv7hvd/SDwZ+DSo3gdEQnRwcZm/vMvy/nd6xu4+vRcfveFUwMJB4CBvbtx+4XjWfjNC7j3iqn07taF//vXQs740av8z4IiSnfvD+R928Pd+fnLa/jB80V8YvIQHrg2P2nC4XDaW+UdwF3A0+6+yszGAK8d5XveZmbXEDlM9Z/uvrvV/GHA1hbPS4HTj/K9RCQEtfWN3PrHJby5roqvXTyBL58/LiFDVDO6pHHZKcO4dOpQlm7Zzdy3SnjgrU384c2NzMzL4foZo8kf2T9hw2Wbm53//tsqHnlnM1fkj+BHn5mcdP2WDqVdAeHurwOvA5hZGlDl7l85ivf7LfB9wKPffwbc0GqZeFuvzeNgZjYHmAOQm5t7FCWJSEeqqKnj+ocWU1xew92fncK/5Y9IeA1mxqkjB3DqyAFs23OAR9/ZzPxFW1iwspzJwzK5fsYoPjllKBldghv91NDUzDf+dwVPv7+NOeeO4a5Zk1LuOo52bR0ze8zM+kZHM60G1pjZ14/0zdx9h7s3uXsz8Acih5NaKwVafqKGA9sP8Zr3u3u+u+dnZ2cfaUki0oE2VNbymd+8zaaqfTxwbX4o4dDasH49uHPWJN656wJ++Ok8DjQ08dUnljPjJ//gl39fR1VtfYe/Z11DE7f+cSlPv7+Nr18yMSXDAdp/iOlEd682s6uBBcA3gSXA3UfyZmY2xN3Lok8/DRTGWWwxMN7MRgPbgCuBzx/J+4hI4i3dspsbH15Mmhnzv3QGJ4/oF3ZJ/6JnRheuPn0kn5+ey5vrqpi7cBO/+Ptafv3aei4+aTAfm5DNOeOzyck8tmGntfWN3PTIYt7btIvvX5bHF88Y2UE/QeK1NyC6mllX4DLgPndvMLNDDn8ys/nAeUCWmZUC3wHOM7OpRA4ZlQA3R5cdSmQ462x3bzSz24CXiAxznevuq470BxOR9nmtuILCbXvp3b0Lvbt1oU/3LvTu1jXyvXsX+nSLfO/RNb3Nv4JfWb2D2+cvZXDf7sy7YTojk+zGNy2ZGedOyObcCdlsqKzlkbdLWLCynOdWRP52HTeoN2ePy+Kc8VmcPmYgvbu1/4Tyrn0Hue6hRazaXs29V0zl0qnDgvoxEqK9w1y/QmSvYTnwCSAX+KO7nxNseUdGw1xF2q+iuo5vP7OKF1eVt2v59DSjd7eWIRIJjm5d0nhl9Q7yhmUy97rTyOrdLeDKO567U1xew1vrqnhzfRWLNu2krqGZLmnGtNz+nD0+i7PHZzFlWGabV22X763jiw++x5Zd+/nN1dO48ITBCf4pjk4grTbMrIu7Nx5TZR1MASFyeO7OEwVb+eHzRdQ1NnPHx8dz/VmjqWtoora+kZq6xuj3hn95XlsXmVYTfVxb3/jB/MnDMvmfz0ym1xH8tZ3M6hqaWLp5N2+ur+KtdVUUbt+LO/Tp3oWzxg7k7PHZnDMui5EDe2JmlFTt4wsPvsee/Q08cG0+Z4wZGPaP0G4dcaFcJpFDROdGJ70OfM/d93ZYlR1AASFyaCVV+7jrqZW8s3Enp48ewI8vn8LorOQ9HJQsdu07yNsbImHx5roqtu05AMDw/j04e1wWfy+qoKm5mXk3nM7k4YfvL5VMOiIgniRyQvmR6KQvAie7+2c6rMoOoIAQia+xqZkH39rEz19ZS0Z6Gv/1iRO4In9EUrajSHbuTsnO/by1rpI311XxzoadZPbsysPXn5YSvaBaO1RAtHd/cKy7X97i+X+b2bJjrkxEAle4bS93PrWCwm3VXHziYL5/WR6D+4bbIC6VmRmjs3oxOqsXXzxzFI1NzaSZHZdh296AOGBmZ7v7WwBmNgM4EFxZInKs6hqauPfv6/jDmxsZ0CuD3149jZl5OSk5Hj+ZpVqr8SPR3oC4BZgXPRcBsBu4NpiSRORYvbNhJ3c9tYKSnfu5In8E/zX7hKS8Y5kkt/a22lgOnGxmfaPPq83sDmBFgLWJyBHae6CBH79QxPxFW8kd0JPHbjqds8Z99I5tIu1xRGPS3L26xdOvAvd2aDUictReLCzn288UUlVbz83njuGOj08IrHuqdA7HMmhZBzKlU2toaqagZDf/XFNBQ5OTN6wvU4ZnMjqrd8I6djY3O5t27uPuF9fw4qpyThjSlwevPS3lhlpKcjqWgDh+bmYt0k679x3kn2sreLWogtfXVlJT10hGehppaVDX0AxAz4x0Thral8nD+jF5eF8mDzv20HB3KmvqWbOjhjXlka+1O2pYu6OWAw1NZHRJ4xszJ/Klc8bQ9Tg+aSqJdciAMLMa4geBAT0CqUgkibg7a3fU8mrxDv5RVMHSLbtpdsjq3Y1ZeTlceMJgzh6XRbcuaWyo3MfKbXtZWbqHldv28tiizdQtjIRGr4x0ThqaSd6wTKYMj3wfk9Ur7tDIvQcaWLvjwxAojn7fs//DW8Bn9e7GxJzeXDl9BBMH92HGuKxQb3wvx6ejbrWRjHShnHSEuoYm3tu0i38U7eDV4gpKd0dGdOcN68sFkwZz4aRBTB6Wedhx741NzayvrGVl6V4Kt+1lxba9rN5eTX3jv4bG5OGZpKfZB4FQtrfug9fo3a0LEwb3ZmJOXyYO7s2EnD5MHNyHgSnY70iSUyC9mJKRAkKOVkV1Ha+tiRw6emt9FfsPNtG9axpnj8vmwhMGcf7EQcfcBho+DI0V0dBYGQ0NB8Zl92ZiTh8mDO7DpJw+TMjpw9DM7rpuQQLVEVdSixyXyvfWcedTK/jnmkoAhmZ25/Jpw7nghEGcOWYg3bt27CigLulpTMrpy6ScvnwuejOdxqbmD+aJJBMFhHRaL6ws486nVnIw2tH0kpNymJTTJ+F/sSsYJFkpIKTT2VffyPf+tprHC7YyZXgmv7zyFHU0FYlDASGdyvKte7jj8WWU7NzHl88fyx0fn6BhoSJtUEBIp9DU7Pzu9Q384pW1DOrTjflfOiOlbuoiEgYFhBz3tu05wH88voxFm3bxiSlD+NFlk9W4TqQdAgsIM5sLfBKocPe8VvO+BtwNZLt7VZx1S4AaoAlobGsIlsjhPLdiO//11Eqamp17/u1kLp82TMNGRdopyD2Ih4H7gHktJ5rZCOAiYMth1j8/XniItEdtfSPfeWYVTy4tZeqIfvzyyqmMHKgT0SJHIrCAcPc3zGxUnFm/AL4BPBPUe0vntnTLbu748zJKd+/nKxeM4/YLx+tEtMhRSOg5CDP7FLDN3ZcfZjffgZfNzIHfu/v9h3jNOcAcgNzc3I4sV1JMY1Mzv/nnBn756jpy+nbn8ZvP5LRRA8IuSyRlJSwgzKwn8C3g4nYsPsPdt5vZIOAVMyt29zfiLRgNj/sh0mqjwwqWlLJ1137+4/FlFGzezWVTh/K9y/Lo210nokWORSL3IMYCo4HY3sNwYKmZTXf38pYLuvv26PcKM3samA7EDQiRFwvL+fpflgNw7xVTueyUYSFXJHJ8SFhAuPtKYFDseXSkUn7rE9Fm1gtIc/ea6OOLge8lqk5JLTV1DXztL8sZndWL31w9TS2vRTpQYGfuzGw+8A4w0cxKzezGQyw71MwWRJ8OBt4ys+XAIuB5d38xqDoltf2loJTa+kZ+cFmewkGkgwU5iumqw8wf1eLxdmB29PFG4OSg6pLjR1Oz8/DbJZw6sj8nj+gXdjkixx2N/ZOU9WrRDrbs2s8NM0aHXYrIcUkBISlr7sJNDM3sziUnDQ67FJHjkgJCUtKq7Xt5d+Murj1rlO6nIBIQ/WZJSnpoYQk9uqZz5Wm6OFIkKAoISTlVtfU8u2w7nz11uLqyigRIASEp50/vbuFgUzPXzRgVdikixzUFhKSU+sYmHn13M+dPzGZsdu+wyxE5rikgJKU8t7yMqtp6rtfQVpHAKSAkZbg7cxduYtyg3pwzPivsckSOewoISRmLNu1i1fZqbpgxWneFE0kABYSkjLkLN9GvZ1c+rW6tIgmhgJCUsHXXfl5ZvYPPT8+lR0Z62OWIdAoKCEkJj7xdQpoZXzxzZNiliHQaCghJerX1jTy+eCuzJw9hSGaPsMsR6TQUEJL0/rdgKzX1jdxwtoa2iiSSAkKSWnOz89DbJUzL7cdU3fNBJKEUEJLU/lFcwead+3VhnEgIFBCS1OYu3MSQzO7MzMsJuxSRTkcBIUmruLyatzfs5JozR9FV93wQSbjAfuvMbK6ZVZhZYZx5XzMzN7O4/RLMbKaZrTGz9WZ2Z1A1SnJ76K0SundN46rpI8IuRaRTCvLPsoeBma0nmtkI4CJgS7yVzCwd+DUwCzgRuMrMTgyuTElGO2vreXrZNi6fNpx+PTPCLkekUwosINz9DWBXnFm/AL4BeBurTgfWu/tGdz8I/Bm4NJgqJVk99t4WDjY2c73u+SASmoQe2DWzTwHb3H35IRYbBmxt8bw0Oq2t15xjZgVmVlBZWdlBlUqYDjY2M+/dzXxsQjbjBvUJuxyRTithAWFmPYFvAd8+3KJxprW1t4G73+/u+e6en52dfSwlSpJ4fuV2KmvqdWGcSMgSuQcxFhgNLDezEmA4sNTMWo9fLAVanpUcDmxPSIUSOndn7lsljBvUm3N1zweRUCUsINx9pbsPcvdR7j6KSBBMc/fyVosuBsab2WgzywCuBJ5NVJ0SriWbd7Ny216uO2uU7vkgErIgh7nOB94BJppZqZndeIhlh5rZAgB3bwRuA14CioAn3H1VUHVKcpm7cBOZPbrymWm654NI2LoE9cLuftVh5o9q8Xg7MLvF8wXAgqBqk+RUuns/LxaWM+fcsfTMCOyjKSLtpMtTJWnMe2czZsY1uueDSFJQQEhS2FffyPxFW5iVl8PQfrrng0gyUEBIUnhyaSk1dbrng0gyUUBI6JqbnYcWljB1RD+m5fYPuxwRiVJASOheX1vJpqp92nsQSTIKCAnVjuo6fvbKGnL6dmeW7vkgklQ0llBC4e78ddk2vvPMKg42NfPzz03VPR9EkowCQhKuqraebz29kpdW7WBabj9+9rmpjM7qFXZZItKKAkISasHKMv7PXwuprWvkrlmTuOmcMaSnqaWGSDJSQEhC7Nl/kG8/s4pnl29n8rBMfva5k5kwWK28RZKZAkIC92rRDu58aiW79x3kqxdN4Nbzxup8g0gKUEBIYKrrGvjBc6t5oqCUSTl9eOi608gblhl2WSLSTgoICcRb66r4xv8up7y6ji+fP5avXDiebl3Swy5LRI6AAkI61L76Rn78QjGPvruZMdm9ePLWszhFV0eLpCQFhHSYRZt28bW/LGfr7v3cdPZovnbJRLp31V6DSKpSQMgxq2to4p6X1vDgwk2M6N+Tx+ecyfTRA8IuS0SOkQJCjkpTs1NQsosXCst5obCMHdX1fOGMXO6adQK9uuljJXI80G9yimtsaua1NZWkGUzL7U//XhmBvVdDUzPvbdzFgsIyXl61g6raejK6pHHu+Gx+/rlRzBiXFdh7i0jiBRYQZjYX+CRQ4e550WnfBy4FmoEK4Lro7UZbr1sC1ABNQKO75wdVZ6o6cLCJvyzZyh/e3MjWXQc+mD42uxenjuwf/RrA2OxemB39lcr1jU0sXF/FCyvLeaVoB3v2N9AzI53zJw5iZl4O508aRG/tMYgcl8zdg3lhs3OBWmBei4Do6+7V0cdfAU5091virFsC5Lt71ZG8Z35+vhcUFBxz7cls976DzHtnM4+8U8KufQc5JbcfN587lv49u7Jky26WlOxmyZbd7NnfAEC/nl05Nbc/06KhcfLwfvTIOPSJ4wMHm3h9bSUvFpbxalEFNfWN9OnWhY+fOJiZeTl8bEK2Tj6LHCfMbElbf4QH9qefu79hZqNaTatu8bQXEEw6HYdKd+/ngTc38fjirRxoaOKCSYO45WNjOW1U/w/2EE4fMxCIdErdWLUvEhabd1OweRevFlcA0CXNOGloX6aN7E/+yAGcOrI/OZndqa1v5B/FFbxYWMZrxZUcaGiif8+uzJqcw6y8IZw1bqCuYxDpZALbgwCIBsRzsT2I6LQfAtcAe4Hz3b0yznqbgN1EAuT37n5/e94vjD2IffWN3PLHJTQ1O9NHD2D66AGcMqL/Yf9Kb6+ismruf2Mjzy7fjgGfmjqUm88dy8ScI+tjtHvfQd7fupuCaGgsL91DXUMzAEMzu1O17yAHG5vJ7tONS04azKy8IZw+egBd1BJD5Lh2qD2IhAdEi3l3Ad3d/Ttx5g119+1mNgh4Bbjd3d9o4z3mAHMAcnNzT928eXNH/giH1NTs3PxoAa+tqWTC4D4Ul1fjDl3TjSnD+30QGKeO7E/f7l3b/bruzrsbd/H7NzbwzzWV9MxI56rpudxw9miG9evRIbU3NDWzens1Szbv5v2te8jqncHsyUOYlttf3VVFOpFkDYiRwPPx5rVa7rtArbvfc7j3S/QexI8WFHH/Gxv53qUncc2Zo9h7oIGlm3fz3qZdLNq0kxWle2lsdtIMThjSl+mjB3D66AGcNmoAA3t3+8jrNTU7L68q53dvbGT51j0M7JXB9TNG8YUzRtKvZ3Cjk0Sk8wrlHEQbhYx393XRp58CiuMs0wtIc/ea6OOLge8lsMx2eXzxFu5/YyPXnDmSa84cBUBmj66cP2kQ508aBERO9r6/JRYYu5i/aAsPLSwBYNyg3h8Exikj+rNwQxX3v7GRTVX7GDmwJz+4LI/PnjpcJ4NFJDRBDnOdD5wHZJlZKfAdYLaZTSQyzHUzcEt02aHAA+4+GxgMPB098doFeMzdXwyqzqPxzoadfOvpQs4Zn8W3P3lim8v1yEjnrHFZnBW9PuBgYzMrt+1lUXQP42/LtvPYe1s+WD5vWF/u+/wpzMobosM8IhK6QA8xJVoiDjGVVO3jst8sJKt3N5689Swye7T/3EJrTc1OUVk172/Zzdjs3pw5duAxXbMgInKkkuYQU6rbu7+BGx5ZjAEPXpt/TOEAkJ5m5A3L1D0SRCQpKSDaqaGpmS8/tpStu/bzp5vOYOTAXmGXJCISKAVEO7g73312FW+tr+Luz05Rp1IR6RR0FVQ7PPx2CX96bwu3fGws/5Y/IuxyREQSQgFxGK+tqeD7z63m4hMH841LJoZdjohIwiggDmFNeQ23P/Y+Jwzpy71XTiVNQ09FpBNRQLShqraeGx9ZTM+MdB64Np+eGTpdIyKdi/7Xi6O+sYlbHl1CZU09T9x8JkMyO6b/kYhIKlFAtOLu3PXkSgo27+bXn5/GySP6hV2SiEgodIipld/8cwNPvb+N/7xoAp+YMiTsckREQqOAaOGFlWXc/dIaLps6lNsuGBd2OSIioVJARK0s3ct/PLGMabn9+PHlU9QTSUQ6PQUEUL63jpvmLWZgr278/ov5arEtIoICgv0HG7lp3mJq6xp58Lp8svt89EY+IiKdUacfxZRmxvhBffjqRROYlNM37HJERJJGpw+I7l3T+cUVU8MuQ0Qk6XT6Q0wiIhKfAkJEROJSQIiISFyBBYSZzTWzCjMrbDHt+2a2wsyWmdnLZja0jXVnmtkaM1tvZncGVaOIiLQtyD2Ih4GZrabd7e5T3H0q8Bzw7dYrmVk68GtgFnAicJWZnRhgnSIiEkdgAeHubwC7Wk2rbvG0F+BxVp0OrHf3je5+EPgzcGlQdYqISHwJH+ZqZj8ErgH2AufHWWQYsLXF81Lg9EO83hxgDkBubm7HFSoi0skl/CS1u3/L3UcAfwJui7NIvCZI8fY0Yq93v7vnu3t+dnZ2R5UpItLphXmh3GPA88B3Wk0vBUa0eD4c2N6eF1yyZEmVmW0+ynqygKqjXDcRVN+xUX3HRvUdm2Sub2RbMxIaEGY23t3XRZ9+CiiOs9hiYLyZjQa2AVcCn2/P67v7Ue9CmFmBu+cf7fpBU33HRvUdG9V3bJK9vrYEFhBmNh84D8gys1IiewqzzWwi0AxsBm6JLjsUeMDdZ7t7o5ndBrwEpANz3X1VUHWKiEh8gQWEu18VZ/KDbSy7HZjd4vkCYEFApYmISDvoSuoP3R92AYeh+o6N6js2qu/YJHt9cZl7mwOERESkE9MehIiIxKWAEBGRuDpVQByuCaBF/Co6f4WZTUtwfSPM7DUzKzKzVWb273GWOc/M9kYbHi4zs4/0swq4xhIzWxl974I480PbhmY2scV2WWZm1WZ2R6tlErr92mhaOcDMXjGzddHv/dtYN/CmlW3Ud7eZFUf//Z42s35trHvIz0KA9X3XzLa1+Dec3ca6YW2/x1vUVmJmy9pYN/Dtd8zcvVN8ERkyuwEYA2QAy4ETWy0zG3iByNXcZwDvJbjGIcC06OM+wNo4NZ4HPBfidiwBsg4xP9Rt2OrfuxwYGeb2A84FpgGFLab9FLgz+vhO4Cdt1H/Iz2uA9V0MdIk+/km8+trzWQiwvu8CX2vHv38o26/V/J8B3w5r+x3rV2fag2hPE8BLgXke8S7Qz8yGJKpAdy9z96XRxzVAEZHeVKkk1G3YwoXABnc/2ivrO4THaVpJZBs9En38CHBZnFUT0rQyXn3u/rK7N0afvkukm0Eo2th+7RHa9osxMwM+B8zv6PdNlM4UEPGaALb+z7c9yySEmY0CTgHeizP7TDNbbmYvmNlJia0MB142syXRRomtJcs2vJK2fzHD3H4Ag929DCJ/FACD4iyTLNvxBiJ7hPEc7rMQpNuih8DmtnGILhm23znADv+we0RrYW6/dulMAdGeJoBH1CgwKGbWG3gSuMP/tUU6wFIih01OBv4f8NcElzfD3acRuV/Hl83s3FbzQ9+GZpZBpJXLX+LMDnv7tVcybMdvAY1EGmvGc7jPQlB+C4wFpgJlRA7jtBb69gOu4tB7D2Ftv3brTAHRniaAR90osKOYWVci4fAnd3+q9Xx3r3b32ujjBUBXM8tKVH0eueodd68AniayK99S6NuQyC/cUnff0XpG2NsvakfssFv0e0WcZULdjmZ2LfBJ4GqPHjBvrR2fhUC4+w53b3L3ZuAPbbxv2NuvC/AZ4PG2lglr+x2JzhQQHzQBjP6FeSXwbKtlngWuiY7EOQPYGzsUkAjRY5YPAkXu/vM2lsmJLoeZTSfyb7gzQfX1MrM+scdETmYWtlos1G0Y1eZfbmFuvxaeBa6NPr4WeCbOMu35vAbCzGYC3wQ+5e7721imPZ+FoOpreU7r0228b2jbL+rjQLG7l8abGeb2OyJhnyVP5BeRETZriYxu+FZ02i3ALdHHRuR2pxuAlUB+gus7m8hu8ApgWfRrdqsabwNWERmV8S5wVgLrGxN93+XRGpJxG/Yk8h9+ZotpoW0/IkFVBjQQ+av2RmAg8CqwLvp9QHTZocCCQ31eE1TfeiLH72Ofwd+1rq+tz0KC6ns0+tlaQeQ//SHJtP2i0x+OfeZaLJvw7XesX2q1ISIicXWmQ0wiInIEFBAiIhKXAkJEROJSQIiISFwKCBERiUsBIXIEzKzJ/rVjbId1CTWzUS27goqELbB7Uoscpw64+9SwixBJBO1BiHSAaG//n5jZoujXuOj0kWb2arSx3KtmlhudPjh6r4Xl0a+zoi+VbmZ/sMj9QF42sx6h/VDS6SkgRI5Mj1aHmK5oMa/a3acD9wH3RqfdR6T9+RQiTe9+FZ3+K+B1jzQNnEbkalqA8cCv3f0kYA9weaA/jcgh6EpqkSNgZrXu3jvO9BLgAnffGG24WO7uA82sikgriIbo9DJ3zzKzSmC4u9e3eI1RwCvuPj76/JtAV3f/QQJ+NJGP0B6ESMfxNh63tUw89S0eN6HzhBIiBYRIx7mixfd3oo/fJtJJFOBq4K3o41eBWwHMLN3M+iaqSJH20l8nIkemR6ub0L/o7rGhrt3M7D0if3hdFZ32FWCumX0dqASuj07/d+B+M7uRyJ7CrUS6gookDZ2DEOkA0XMQ+e5eFXYtIh1Fh5hERCQu7UGIiEhc2oMQEZG4FBAiIhKXAkJEROJSQIiISFwKCBERiev/A7+HCOPIByHIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"train accuracy\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkpklEQVR4nO3de3hddZ3v8fc3O/c0yW6bXpImvUNLgTaUytUCYhFQEAcVRB3Qg4M4OqNnvD8+x9tczqijOKgHRQVkZuRwEGcQpCgg06oUJYVe6IXeaGlomqRtbs398j1/7JWyG5ImbbL32jv5vJ5nPdn7t9be68vqJp+s32+t3zZ3R0RE5EQywi5ARERSn8JCRESGpbAQEZFhKSxERGRYCgsRERlWZtgFJEpJSYnPnTs37DJERNLK+vXrD7n7tIHt4zYs5s6dS1VVVdhliIikFTPbN1i7uqFERGRYCgsRERmWwkJERIalsBARkWEpLEREZFgJCwszu8fM6szspbi2B81sQ7DsNbMNQXu2md1rZpvNbKOZXRb3mnOD9l1mdqeZWaJqFhGRwSXyzOI+4Kr4Bne/0d0r3b0SeBj4ZbDqr4L1ZwNXAN82s/7a7gJuA04LluPeU0REEi9hYeHua4Ejg60Lzg5uAB4ImpYATwevqwMagRVmVgoUufs6j82lfj/wrgTWzL89t4/HNh1I1C5ERNJSWGMWK4Fad98ZPN8IXGdmmWY2DzgXqABmAdVxr6sO2gZlZreZWZWZVdXX1590UWbGL6r286M1e076tSIi41lYYXETr59VANxDLAiqgO8CzwI9wGDjE0N+W5O73+3uK9x9xbRpb7hbfUSuXVbG5tea2Huo9ZReLyIyHiU9LMwsE7geeLC/zd173P1/BuMZ1wFRYCexACmPe3k5kNA+orefXQqgrigRkThhnFmsAra7+7HuJTPLN7OC4PEVQI+7b3X3GqDFzC4IxjluBh5JZHFl0TzeNHcyj26sSeRuRETSSiIvnX0AWAcsMrNqM7s1WPU+ju+CApgOvGBm24DPA38Zt+5jwE+AXcBuYHWiau537bIyXq5t4eWDLYnelYhIWkjYrLPuftMQ7R8apG0vsGiI7auAs8aytuFcfVYpX/3VFh7bdIBFMwctS0RkQtEd3IOYVpjDhQum8timGmJX7IqITGwKiyFcu7SMVw61suVAc9iliIiETmExhKvOmklmhvHoRl0VJSKisBhCND+bS06fpq4oEREUFid0zdJSXmts54VXG8MuRUQkVAqLE7hiyQyyMzPUFSUiE57C4gQKc7O4fNF0fr25ht4+dUWJyMSlsBjGtcvKqG/p5E+vHA67FBGR0CgshnH54unkZ0d4bJOm/xCRiUthMYy87AirzpjB6s01dPf2hV2OiEgoFBYjcO2yMhrauvnjrkNhlyIiEgqFxQhccnoJhbmZmolWRCYshcUI5GRGuPLMmfx2y0E6e3rDLkdEJOkUFiN07bIyWjp7WPPyyX9dq4hIulNYjNBFC6YypSCbR3VVlIhMQAqLEcqKZHD1WTN5amstbV09YZcjIpJUCouTcM3SMtq7e/nd9rqwSxERSSqFxUk4b94UphfmaK4oEZlwFBYnIZJhvGNpKc+8XE9zR3fY5YiIJI3C4iRdu6yMrp4+ntxSG3YpIiJJo7A4SedURJkVzeOxTeqKEpGJQ2FxksyMa5aV8vudh2ho7Qq7HBGRpFBYnIJrl5bR0+c8seVg2KWIiCSFwuIUnFlWxPySAl0VJSIThsLiFJgZ1ywt5bk9h6lr6Qi7HBGRhFNYnKJrl5XR57B6s7qiRGT8U1icotNmFLJ4ZqG6okRkQlBYjMK1y8qo2tfAgcb2sEsREUkohcUoXLO0FIBfayZaERnnFBajMGdqAUvLi3lUN+iJyDinsBila5eWsam6ib2HWsMuRUQkYRQWo/SOoCtK03+IyHimsBilsmgeK+ZM5jGNW4jIOKawGAPXLitj+8EWdtS2hF2KiEhCKCzGwNVnzyTD4DHdcyEi45TCYgxML8zlwgVTeXRTDe4edjkiImNOYTFGrllaxiuHWtlyoDnsUkRExlzCwsLM7jGzOjN7Ka7tQTPbECx7zWxD0J5lZj8zs81mts3Mvhj3mnOD9l1mdqeZWaJqHo2rzpxJZobpngsRGZcSeWZxH3BVfIO73+jule5eCTwM/DJY9V4gx93PBs4FPmpmc4N1dwG3AacFy3HvmSomF2Sz8rQSHtuorigRGX8SFhbuvhY4Mti64OzgBuCB/s2BAjPLBPKALqDZzEqBIndf57HfwPcD70pUzaN17bIyXmts54VXG8MuRURkTIU1ZrESqHX3ncHzXwCtQA3wKvAv7n4EmAVUx72uOmgblJndZmZVZlZVX1+fmMpP4IolM8jOzODutbtp7+pN+v5FRBIlrLC4idfPKgDOA3qBMmAe8Gkzmw8MNj4xZB+Pu9/t7ivcfcW0adPGst4RKczN4q8vW8BvttTy9jt/T9XeQU+sRETSTtLDIuhquh54MK75/cAT7t7t7nXAH4EVxM4kyuO2KwdSegT5U6tO5+cfOZ/u3j7e+6N1/P1jW9PiLMPdeeblOt5z17Ncecda/nn1dtbvO0Jvn8ZfRCScM4tVwHZ3j+9eehW43GIKgAuCbWqAFjO7IBjnuBl4JPkln5yLFpbwm09dwgfPn8NP//AKV//rWp5P0bMMd2ftjnquv+tZPnzv8xxs7mDqpGx+8vs9vPuudbzpH5/iMw9t5ImXamjt7Am7XBEJiSXqyh0zewC4DCgBaoGvuPtPzew+4Dl3/2HctpOAe4ElxLqe7nX3bwXrVhC7sioPWA38jY+g6BUrVnhVVdVY/iedkmd3H+LzD2+iuqGdD100l89duZi87EjYZeHuPLv7MHc8uYOqfQ3MiubxicsX8u7l5WRnZtDU3s2aHfU8va2WZ7bX0dzRQ3YkgwsXTGXVkhm8dfF0yqJ5Yf9niMgYM7P17r7iDe3j9TLPVAkLgNbOHr75xHZ+tm4fc6bm8633LOO8eVNCq+e5PYf5zpM7+PMrR5hZlMvHL1/IDSvKyckcPMS6e/uo2tvA09tqeWpbLXsPtwGwpLSIVUtmsOqM6ZxVVkxGRkreAiMiJ0FhkQLW7T7M5x/exP6GNm65cC6fu2oR+dmZSdv/83uPcMeTO3h292GmF+bw8bcs5MY3VZCbNfIzHXdnd30rT22r5elttazf10Cfw4yiHC5fPIMrlkznogUlJ/WeIpI6FBYpoq2rh28+8TL3PbuXOVPz+ea7l3L+/KkJ3ef6fQ1896kd/H7nIUom5fDXly3g/efPHpNf6Edau3hmex1Pbatl7Y56Wrt6qZiSx/dvWs6yiujoixeRpFJYpJjn9hzmc7/YxKtH2mJjGQk4y9iwv5E7ntzBmh31TC3I5vZLF/DBC+YkbMyks6eX3+84xJcfeYn6o5188eoz+PDFc0nRGVpEZBAKixQUf5Yxe0o+33zPUi4Yg7OMzdVN3PHUDn63vY7J+Vl89NIF3HzhnKR1eTW2dfGZhzby1LY63rZkBt96zzKK87OSsm8RGR2FRQr7057DfO7hTew73MYtF87hLy+cS0d3L21dvbR19dDe1UtrVy/tXT20dgXtnT20dQc/u17ftqWjh511RynOy+K2S+Zzy0VzmZSTvHGRfu7OT//wCv+8ejszi3P5/vuXU6luKZGUp7BIce1dvXzrNy9z77OvMJJ/kvzsSLBkHvc4LztCZUWUmy+cQ2Fu+H/Nv/hqA5/4+YvUtXTw+asWc+ub56lbSiSFKSzSxEuvNbGr7ih52REKgl/+BTkR8rMyyc+JhUJuZiStLlNtauvmM7/YyJNba1l1xgz+5b1LieZnh12WiAxCYSGhcnfu+eNe/nn1NqYX5vK995/D8tmTwy5LRAYYKiz0TXmSFGbGrW+ex0O3X4QZ3PDDdfx47R5994dImlBYSFJVVkT59d+u5K1nTOcfH9/GR35WRUNrV9hlicgwFBaSdMV5Wfzwg+fylWuXsHZnPe+48/es39cQdlkicgIKCwmFmfHhi+fxi9svIhIxbvzROn60Zjd9mhJdJCUpLCRUyyqiPPY3K1l1xgz+9+rtfOT+Ko6oW0ok5SgsJHTFeVnc9cHlfO2dZ/KHnYd48zd+x2cf2sifXzmiAXCRFJH8W3tFBmFm3HLRXM6fP4V7/7CXxzYd4KH11cwrKeA955bz7uXlzCzODbtMkQlL91lISmrt7OHxzTU8tL6aP79yhAyDS06fxnvPrWDVkulDfveGiIyObsqTtLX3UCu/WF/Nwy9UU9PUQTQ/i3dVzuK9K8o5s6w47PJExhWFhaS93j7nD7sO8f+q9vPkllq6evtYUlrEDSvKua5yFpMLNIWIyGgpLGRcaWzr4pENB3ho/X5eeq2Z7EgGVyyZwXtWlHPJadOIpNHcWSKpRGEh49bWA808tH4///XiazS0dTO/pICvvPNMLj19WtiliaQdhYWMe509vTy5tZZv/3YHrxxq5cozZ/C/rllC+eT8sEsTSRuaSFDGvZzMCNcsLeOJT63ks1cuYu2OQ6z6zhq+9/ROOrp7wy5PJK0pLGTcycmM8PG3LOSpT1/KWxZN59tP7uCq767lmZfrwi5NJG0pLGTcmhXN464Pnsu/3XoeGRnGh+99nr+6v4r9R9rCLk0k7SgsZNxbedo0nvjkJXz+qsX8cVesa+pfn1LXlMjJUFjIhJCdmcHHLlvA05++lFVLZnDHUzt42x1reXpbbdiliaQFhYVMKKXFefzg/cv5j4+cT1bEuPVnVdx63/O8elhdUyInorCQCenihSWs/uQlfPHqxazbc5hVd6zhjid3qGtKZAgKC5mwsjMz+OilC/jdpy/jyjNn8q9P7+SKO9bw4POv8sqhVk2PLhJHN+WJBJ7dfYivPLKFnXVHAYjmZ7GsPMo5s6NUVsSWaL7mn5LxTXdwi4xAb5+zs66FDa828uKrjWzY38iOuhb6/zeZV1LAORVRKoMAWTyziOxMnaDL+KGwEDlFRzt72FT9enhs2N9IfUsnADmZGZw1q5jKitfPQGZF8zDTRIaSnhQWImPE3XmtsT0WHEGAbH6tic6ePgDysyMU52VRnJdFUW4WRf2P8zKPtR1bnxf/OJO8rIiCRkI1VFjoa1VFTpKZUT45n/LJ+VyztAyA7t4+tte0sGF/A3sPt9Hc3k1TezfNHd281tjOtppmmtu7aensOeF7Z0WMyfnZzCzOZUZRLjOLcplRlBN7XBw8L86lMCdToSJJpbAQGQNZkQzOLi/m7PITf3NfT28fRzt7aOoPk/aeY6HS33bkaBcHmzvYf6SN5/ceobGt+w3vk58dYWZRLtOLco4FyMwgXKL52WRFjMxIBpkZRmbEyMwY8DhiwfOgPcOIZJgCSIaksBBJosxIBtH87JO6qqqju5fa5g5qmzs52NxBbVMHB5s7jj2u2tdAXXMnXb19o68vw5iUm0nF5HwqpuRRMSWfisn5zJ4SW8qieRrQn6AUFiIpLjcrwpypBcyZWjDkNu5OQ1s3B5s6aGzvorfP6el1evqcnt6+2M++Prp7PVgXtMVt093n9Pb10dTezf4j7WyvaeGprXXHhVCGxe6Cr5iSdyxEKoJl9pR8SiZl6+xknEpYWJjZPcA1QJ27nxW0PQgsCjaJAo3uXmlmHwA+G/fypcByd99gZucC9wF5wOPAJ328jsqLnCIzY0pBNlPG+HvI+/qc2pYOXj3cxv6Gdl490sb+YFmzo5664KqwfnlZEUqjuZQU5DClIJupk7KZWpDN1EnB87jHk/OzyIzoLCVdJOxqKDO7BDgK3N8fFgPWfxtocvevD2g/G3jE3ecHz/8MfBJ4jlhY3Onuq4fbv66GEkm8ju5eqhvaghCJhUlNUzuHj3ZxuLWLI61dNLR1MdivGTOI5mUFoZITBEk2RblZZGdmxJZIBlnBkp2ZQVbEjrXFnmeQnWnHPY/mZTF1Uk7yD8Y4MaqrocysAGh39z4zOx1YDKx29zeOvAXcfa2ZzR3i/Qy4Abh8kNU3AQ8E25UCRe6+Lnh+P/AuYNiwEJHEy82KsHB6IQunFw65TW+f09AWC45DRzs50tr/uIsjrZ3HgmVHbQtH9nTR3NFDb9/o/ogtLc5laXkxS8ujsZ+zohTnZ43qPSe6kXZDrQVWmtlk4GmgCrgR+MAp7nclUOvuOwdZdyNwXfB4FlAdt646aBuUmd0G3AYwe/bsUyxNRMZSJMMomZRDyaQcTp8xdKjE6+1zunv76O7to6snNtbS3dtH13FtfXT1HL9dV28f9S2dbH6tiU3VTfxmy+tT0M+Zms/S8ijLyos5e1YxZ80qpiBHw7YjNdIjZe7eZma3At9z92+a2Yuj2O+xs4fjdmJ2PtDm7i/1Nw3y2iH/5HD3u4G7IdYNNYr6RCREkQwjkhEhNysyqvdpauvmpQNNbKxuZNP+Jl7Y18CjGw8AsW6whdMmxQKkIhYgZ5QWjXqf49WIw8LMLiR2JnHrSb524BtlAtcD5w6y+n0cHyLVQHnc83LgwKnsV0QmnuL8LC5eWMLFC0uOtcXOPBrZVB07+1izo46HX4h1YGRmGPNKCpgcDMBH87KJFmQxOT+baF4W0fygPe7nRLmUeKS/8D8FfBH4T3ffYmbzgWdOcZ+rgO3uHt+9hJllAO8FLulvc/caM2sxswuAPwE3A987xf2KiDCtMIfLF8/g8sUzgNhlxzVNHWyqbmRjdRN76o/S0NbN3kNtNLQ10tjWfcJ7WPKzI7Ewyc8imp9FZUWUT1+xiIyM8XUJ8YjCwt3XAGvg2C/1Q+7+tyd6jZk9AFwGlJhZNfAVd/8pbzx76HcJUO3uewa0f4zXL51djQa3RWQMmRll0TzKonlcdVbpG9a7O+3dvTS0ddPQ2kVTezcNbV00tHXT2NpFY/C8sa2b2uYOfvDMbuaXTOLd55YPsrf0NaJLZ83s58DtQC+wHigGvuPu30pseadOl86KSLL19Tnv/uGz7D/Szu8+cylFuel3BdZQl86OtLNtibs3E7ts9XFgNvCXY1eeiEj6y8gwvv7Oszjc2skdT+4Iu5wxNdKwyDKzLGJh8Uhwf4WuNhIRGeDs8mLef95s7l+3j+0Hm8MuZ8yMNCx+BOwFCoC1ZjYHGD9HQURkDH3mbYsozM3ky49sGTff5T6isHD3O919lru/3WP2AW9JcG0iImlpckE2n7tyMX9+5Qi/2jg+rvYfUViYWbGZfcfMqoLl28TOMkREZBA3vqmCpeXF/NPj2zg6zJdepYORdkPdA7QQm8/pBmJdUPcmqigRkXQXyTC+9s4zqW3u5M6nB5vZKL2MNCwWuPtX3H1PsHwNmJ/IwkRE0t05sydz44oK7vnDK+yqawm7nFEZaVi0m9mb+5+Y2cVAe2JKEhEZPz531SLysyN85VfpPdg90rC4HfiBme01s73A94GPJqwqEZFxYuqkHD5z5SL+uOswj28+GHY5p2ykV0NtdPdlxL7Bbqm7n8Pg30UhIiIDfOD8OSwpLeIffr2Vtq70HOw+qekS3b05uJMb4O8SUI+IyLgTyTC+ft2Z1DR18P3f7Qq7nFMymrl1x9eUiiIiCbRi7hSuXz6LH/9+D3vqj4ZdzkkbTVik70iNiEgIvnD1YnIzI3z10a1pN9h9wrAIvkuieZClBShLUo0iIuPC9MJcPnXF6azdUc9vt9YO/4IUcsKwcPdCdy8aZCl0d315rYjISbrlwjksmlHI1x/dSkd3b9jljNjE+D5AEZEUkRnJ4GvXnclrje38n//eHXY5I6awEBFJsgvmT+Wdy8r44Zrd7DvcGnY5I6KwEBEJwZfecQZZGcbfP7Y17FJGRGEhIhKCGUW5fHLVaTy1rY7fbU/9wW6FhYhISD500TwWTCvgq79K/cFuhYWISEiyMzP4+nVn8eqRNn68dk/Y5ZyQwkJEJEQXLyzhHWeX8oP/3kV1Q1vY5QxJYSEiErIvveMMjNQe7FZYiIiErCyaxycuX8hvttSyZkd92OUMSmEhIpICPrJyHvNKCvjao6n5JUkKCxGRFJCTGeGvVs5nT30rrxxKvRv1FBYiIili+ZwoABv2N4Zax2AUFiIiKeK06YUUZEfYqLAQEZGhRDKMs8uLdWYhIiIntqwiytaa5pS7o1thISKSQs6piNLd62yraQ67lOMoLEREUkhlxWQg9Qa5FRYiIilkZnEuM4pyFBYiInJilRVRhYWIiJxYZcVk9h1uo6G1K+xSjlFYiIikmMqKKAAbqhtDrSOewkJEJMWcXV6MGWx4tTHsUo5JWFiY2T1mVmdmL8W1PWhmG4Jlr5ltiFu31MzWmdkWM9tsZrlB+7nB811mdqeZWaJqFhFJBZNyMjl9eiEbJ8iZxX3AVfEN7n6ju1e6eyXwMPBLADPLBP4duN3dzwQuA7qDl90F3AacFizHvaeIyHhUWRFl4/7GlJmBNmFh4e5rgSODrQvODm4AHgia3gZscveNwWsPu3uvmZUCRe6+zmNH7H7gXYmqWUQkVSyriNLQ1s2+w6nx7XlhjVmsBGrdfWfw/HTAzew3ZvaCmX0uaJ8FVMe9rjpoExEZ1/oHuVOlKyqssLiJ188qADKBNwMfCH7+hZm9FRhsfGLIczIzu83Mqsysqr4+Nb9tSkRkJE6fMYm8rAgvpsggd9LDIhifuB54MK65Gljj7ofcvQ14HFgetJfHbVcOHBjqvd39bndf4e4rpk2bNvbFi4gkSWYkI6VmoA3jzGIVsN3d47uXfgMsNbP8IEwuBba6ew3QYmYXBOMcNwOPJL9kEZHkq6yIsvVAM5094c9Am8hLZx8A1gGLzKzazG4NVr2P47ugcPcG4DvA88AG4AV3/3Ww+mPAT4BdwG5gdaJqFhFJJZUVUbp6+9he0xJ2KWQm6o3d/aYh2j80RPu/E7t8dmB7FXDWmBYnIpIGjt3Jvb+RZcHjsOgObhGRFFVanMu0wtSYgVZhISKSoszs2M15YVNYiIiksMqKKHsOtdLU1j38xgmksBARSWGpMgOtwkJEJIUtDWagDbsrSmEhIpLCCnOzWDhtUuiD3AoLEZEU1/81q2HOQKuwEBFJccsqohxp7aK6oT20GhQWIiIprn+Q+8UQu6IUFiIiKW7RzEJyszJC/ZpVhYWISIrLimRwVlkxG/Y3hFaDwkJEJA1UVkR56UAz3b19oexfYSEikgYqZ0fp6glvBlqFhYhIGnh9BtpwuqIUFiIiaWBWNI+SSdls2N8Uyv4VFiIiaaB/BlqdWYiIyAlVVkTZXd9KU3vyZ6BVWIiIpIn+b8vbXJ38riiFhYhImlhaHgXCGeRWWIiIpInivCwWTCsIZQZahYWISBpZFtIMtAoLEZE0ck5FlENHu3itMbkz0CosRETSSGXFZICkd0UpLERE0sji0kKyM5M/A63CQkQkjcRmoC1iY3VjUversBARSTOVFZPZ/FpTUmegVViIiKSZytlROrr7ePlg8magVViIiKSZyuDmvGR2RSksRETSTMWUPKYUZCd1kFthISKSZl6fgbYxaftUWIiIpKHKiii76o/S0pGcGWgVFiIiaWhZRRT35M1Aq7AQEUlD/YPcLyapK0phISKShorzs5hfkrwZaBUWIiJpKpkz0CosRETSVGVFlPqWTmqaOhK+L4WFiEiaqgy+ZjUZXVEKCxGRNLW4tJDsSAYb0zkszOweM6szs5fi2h40sw3BstfMNgTtc82sPW7dD+Nec66ZbTazXWZ2p5lZomoWEUknOZkRlpQVJeWKqESeWdwHXBXf4O43unulu1cCDwO/jFu9u3+du98e134XcBtwWrAc954iIhNZZUWUzdVN9CR4BtqEhYW7rwWODLYuODu4AXjgRO9hZqVAkbuv89hw//3Au8a4VBGRtHXO7Cjt3b3sqD2a0P2ENWaxEqh1951xbfPM7EUzW2NmK4O2WUB13DbVQdugzOw2M6sys6r6+vqxr1pEJMUsS9IMtGGFxU0cf1ZRA8x293OAvwN+bmZFwGDjE0NeUOzud7v7CndfMW3atDEtWEQkFc2Zms/k/KyEz0CbmdB3H4SZZQLXA+f2t7l7J9AZPF5vZruB04mdSZTHvbwcOJC8akVEUpuZHbs5L5HCOLNYBWx392PdS2Y2zcwiweP5xAay97h7DdBiZhcE4xw3A4+EULOISMpaVh5lR10LRzt7EraPRF46+wCwDlhkZtVmdmuw6n28cWD7EmCTmW0EfgHc7u79g+MfA34C7AJ2A6sTVbOISDqqnJ34GWgT1g3l7jcN0f6hQdoeJnYp7WDbVwFnjWlxIiLjSP8MtBv2N3LhgqkJ2Yfu4BYRSXOTC7KZOzU/oXdyKyxERMaBRA9yKyxERMaByoooB5s7OJigGWgVFiIi40CiZ6BVWIiIjANnlBaRFTGFhYiIDC03K8KS0iI27G9IyPsrLERExon+GWh7+8b+a1aTPt2HiIgkxsrTptHc0cPRzh6K87LG9L0VFiIi48SqJTNYtWRGQt5b3VAiIjIshYWIiAxLYSEiIsNSWIiIyLAUFiIiMiyFhYiIDEthISIiw1JYiIjIsMx97G8LTwVmVg/sO8WXlwCHxrCcsab6Rkf1jY7qG51Ur2+Ou08b2Dhuw2I0zKzK3VeEXcdQVN/oqL7RUX2jk+r1DUXdUCIiMiyFhYiIDEthMbi7wy5gGKpvdFTf6Ki+0Un1+galMQsRERmWzixERGRYCgsRERnWhA4LM7vKzF42s11m9oVB1puZ3Rms32Rmy5NYW4WZPWNm28xsi5l9cpBtLjOzJjPbECxfTlZ9wf73mtnmYN9Vg6wP8/gtijsuG8ys2cw+NWCbpB4/M7vHzOrM7KW4tilm9qSZ7Qx+Th7itSf8rCawvm+Z2fbg3+8/zSw6xGtP+FlIYH1fNbPX4v4N3z7Ea8M6fg/G1bbXzDYM8dqEH79Rc/cJuQARYDcwH8gGNgJLBmzzdmA1YMAFwJ+SWF8psDx4XAjsGKS+y4DHQjyGe4GSE6wP7fgN8m99kNjNRqEdP+ASYDnwUlzbN4EvBI+/AHxjiPpP+FlNYH1vAzKDx98YrL6RfBYSWN9Xgc+M4N8/lOM3YP23gS+HdfxGu0zkM4vzgF3uvsfdu4D/C1w3YJvrgPs95jkgamalySjO3Wvc/YXgcQuwDZiVjH2PodCO3wBvBXa7+6ne0T8m3H0tcGRA83XAz4LHPwPeNchLR/JZTUh97v5bd+8Jnj4HlI/1fkdqiOM3EqEdv35mZsANwANjvd9kmchhMQvYH/e8mjf+Mh7JNglnZnOBc4A/DbL6QjPbaGarzezM5FaGA781s/Vmdtsg61Pi+AHvY+j/ScM8fgAz3L0GYn8gANMH2SZVjuP/IHamOJjhPguJ9Imgm+yeIbrxUuH4rQRq3X3nEOvDPH4jMpHDwgZpG3gd8Ui2SSgzmwQ8DHzK3ZsHrH6BWNfKMuB7wH8lszbgYndfDlwNfNzMLhmwPhWOXzbwTuChQVaHffxGKhWO45eAHuA/hthkuM9CotwFLAAqgRpiXT0DhX78gJs48VlFWMdvxCZyWFQDFXHPy4EDp7BNwphZFrGg+A93/+XA9e7e7O5Hg8ePA1lmVpKs+tz9QPCzDvhPYqf78UI9foGrgRfcvXbgirCPX6C2v2su+Fk3yDZhfw5vAa4BPuBBB/tAI/gsJIS717p7r7v3AT8eYr9hH79M4HrgwaG2Cev4nYyJHBbPA6eZ2bzgr8/3Ab8asM2vgJuDq3ouAJr6uwwSLejj/Cmwzd2/M8Q2M4PtMLPziP17Hk5SfQVmVtj/mNhA6EsDNgvt+MUZ8i+6MI9fnF8BtwSPbwEeGWSbkXxWE8LMrgI+D7zT3duG2GYkn4VE1Rc/BvYXQ+w3tOMXWAVsd/fqwVaGefxOStgj7GEuxK7W2UHsSokvBW23A7cHjw34QbB+M7AiibW9mdip8iZgQ7C8fUB9nwC2ELu64zngoiTWNz/Y78aghpQ6fsH+84n98i+Oawvt+BELrRqgm9hfu7cCU4GngZ3BzynBtmXA4yf6rCapvl3E+vv7P4M/HFjfUJ+FJNX3b8FnaxOxAChNpeMXtN/X/5mL2zbpx2+0i6b7EBGRYU3kbigRERkhhYWIiAxLYSEiIsNSWIiIyLAUFiIiMiyFhcgpMrNeO35m2zGbzdTM5sbPXioStsywCxBJY+3uXhl2ESLJoDMLkTEWfDfBN8zsz8GyMGifY2ZPB5PePW1ms4P2GcF3RWwMlouCt4qY2Y8t9n0mvzWzvND+o2TCU1iInLq8Ad1QN8ata3b384DvA98N2r5PbMr2pcQm5LszaL8TWOOxCQ2XE7uLF+A04AfufibQCLw7of81IiegO7hFTpGZHXX3SYO07wUud/c9wWSQB919qpkdIjYdRXfQXuPuJWZWD5S7e2fce8wFnnT304Lnnwey3P0fkvCfJvIGOrMQSQwf4vFQ2wymM+5xLxpjlBApLEQS48a4n+uCx88Sm/EU4APAH4LHTwMfAzCziJkVJatIkZHSXyoipy7PzDbEPX/C3fsvn80xsz8R+4PspqDtb4F7zOyzQD3w4aD9k8DdZnYrsTOIjxGbvVQkZWjMQmSMBWMWK9z9UNi1iIwVdUOJiMiwdGYhIiLD0pmFiIgMS2EhIiLDUliIiMiwFBYiIjIshYWIiAzr/wMJfPDGulGwBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"train loss\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy%:  18.02 == 1802 / 10000\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    \n",
    "    y_true_test = []\n",
    "    y_pred_test = []\n",
    "    \n",
    "    for batch_idx, (img, labels) in enumerate(testloader):\n",
    "        img = img.to(device)\n",
    "        label = label.to(device)\n",
    "    \n",
    "        preds = model(img)\n",
    "        \n",
    "        y_pred_test.extend(preds.detach().argmax(dim=-1).tolist())\n",
    "        y_true_test.extend(labels.detach().tolist())\n",
    "        \n",
    "    total_correct = len([True for x, y in zip(y_pred_test, y_true_test) if x==y])\n",
    "    total = len(y_pred_test)\n",
    "    accuracy = total_correct * 100 / total\n",
    "    \n",
    "    print(\"Test Accuracy%: \", accuracy, \"==\", total_correct, \"/\", total)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}